package com.bfmet.crawler;

import org.htmlparser.Node;
import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.filters.AndFilter;
import org.htmlparser.filters.HasAttributeFilter;
import org.htmlparser.filters.TagNameFilter;
import org.htmlparser.tags.Div;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.util.NodeList;

import com.bfmet.entity.News;
import com.bfmet.entity.News.Type;

public class IndustrySpider extends Spider {

	@Override
	public void run() {
		this.getPageList("http://www.asianmetal.cn/news/getMoreProNews.am?columnSize=8&jspBarNewsType=industry");
	}

	private void getPageList(String url) {
		this.log("Fetch industry begin..." + url);
		try {
			Parser p = new Parser(url);
			NodeList liList = p.parse(new AndFilter(new TagNameFilter("li"),
					new HasAttributeFilter("class", "newsTitleB")));
			for (Node node : liList.toNodeArray()) {
				if (node.getChildren().elementAt(1) instanceof LinkTag) {
					LinkTag a = (LinkTag) node.getChildren().elementAt(1);
					this.getIndustryContent(a.getLink());
				}
			}

		} catch (Exception ex) {
			this.logException(ex);
		}
		log("Fetch industry end..." + url);
	}

	private void getIndustryContent(String subUrl) {
		try {
			log("Fetch industry subUrl begin..." + subUrl);
			News news = new News();
			NodeFilter divf = new TagNameFilter("div");

			Parser p = new Parser(subUrl);
			NodeList nodeList = p.parse(new AndFilter(divf,
					new HasAttributeFilter("class", "titleDiv")));
			if (nodeList.elementAt(0) instanceof Div) {
				Div div = (Div) nodeList.elementAt(0);
				String title = div.getFirstChild().getNextSibling()
						.toPlainTextString();
				news.setName(title);

				p.setURL(subUrl);
				NodeList nl = p.parse(new AndFilter(divf,
						new HasAttributeFilter("class", "contentDiv")));
				if (nl.elementAt(0) instanceof Div) {
					Div divContent = (Div) nl.elementAt(0);
					String content = divContent.toPlainTextString();
					news.setContent(content);
				}
			}
			news.setType(Type.INDUSTRY);
			this.save(news);
		} catch (Exception ex) {
			this.logException(ex);
		}
		log("Fetch industry subUrl end..." + subUrl);
	}

	/**
	 * @param args
	 */
	public static void main(String[] args) {
		IndustrySpider is = new IndustrySpider();
		is.run();
	}

}
